{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "%pip install --quiet -U langchain-scrapegraph langchain-google-genai pandas matplotlib seaborn"
      ],
      "metadata": {
        "id": "_RzIaH9bCwM8"
      },
      "id": "_RzIaH9bCwM8",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import getpass\n",
        "import os\n",
        "import json\n",
        "import pandas as pd\n",
        "from typing import List, Dict, Any\n",
        "from datetime import datetime\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns"
      ],
      "metadata": {
        "id": "XXw-4bx8Dl3M"
      },
      "id": "XXw-4bx8Dl3M",
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "if not os.environ.get(\"SGAI_API_KEY\"):\n",
        "    os.environ[\"SGAI_API_KEY\"] = getpass.getpass(\"ScrapeGraph AI API key:\\n\")\n",
        "\n",
        "if not os.environ.get(\"GOOGLE_API_KEY\"):\n",
        "    os.environ[\"GOOGLE_API_KEY\"] = getpass.getpass(\"Google API key for Gemini:\\n\")"
      ],
      "metadata": {
        "id": "7MiibpaPDnTE"
      },
      "id": "7MiibpaPDnTE",
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain_scrapegraph.tools import (\n",
        "    SmartScraperTool,\n",
        "    SearchScraperTool,\n",
        "    MarkdownifyTool,\n",
        "    GetCreditsTool,\n",
        ")\n",
        "from langchain_google_genai import ChatGoogleGenerativeAI\n",
        "from langchain_core.prompts import ChatPromptTemplate\n",
        "from langchain_core.runnables import RunnableConfig, chain\n",
        "from langchain_core.output_parsers import JsonOutputParser\n",
        "\n",
        "smartscraper = SmartScraperTool()\n",
        "searchscraper = SearchScraperTool()\n",
        "markdownify = MarkdownifyTool()\n",
        "credits = GetCreditsTool()\n",
        "\n",
        "llm = ChatGoogleGenerativeAI(\n",
        "    model=\"gemini-1.5-flash\",\n",
        "    temperature=0.1,\n",
        "    convert_system_message_to_human=True\n",
        ")"
      ],
      "metadata": {
        "id": "hislsrbKDpdb"
      },
      "id": "hislsrbKDpdb",
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class CompetitiveAnalyzer:\n",
        "    def __init__(self):\n",
        "        self.results = []\n",
        "        self.analysis_timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
        "\n",
        "    def scrape_competitor_data(self, url: str, company_name: str = None) -> Dict[str, Any]:\n",
        "        \"\"\"Scrape comprehensive data from a competitor website\"\"\"\n",
        "\n",
        "        extraction_prompt = \"\"\"\n",
        "        Extract the following information from this website:\n",
        "        1. Company name and tagline\n",
        "        2. Main products/services offered\n",
        "        3. Pricing information (if available)\n",
        "        4. Target audience/market\n",
        "        5. Key features and benefits highlighted\n",
        "        6. Technology stack mentioned\n",
        "        7. Contact information\n",
        "        8. Social media presence\n",
        "        9. Recent news or announcements\n",
        "        10. Team size indicators\n",
        "        11. Funding information (if mentioned)\n",
        "        12. Customer testimonials or case studies\n",
        "        13. Partnership information\n",
        "        14. Geographic presence/markets served\n",
        "\n",
        "        Return the information in a structured JSON format with clear categorization.\n",
        "        If information is not available, mark as 'Not Available'.\n",
        "        \"\"\"\n",
        "\n",
        "        try:\n",
        "            result = smartscraper.invoke({\n",
        "                \"user_prompt\": extraction_prompt,\n",
        "                \"website_url\": url,\n",
        "            })\n",
        "\n",
        "            markdown_content = markdownify.invoke({\"website_url\": url})\n",
        "\n",
        "            competitor_data = {\n",
        "                \"company_name\": company_name or \"Unknown\",\n",
        "                \"url\": url,\n",
        "                \"scraped_data\": result,\n",
        "                \"markdown_length\": len(markdown_content),\n",
        "                \"analysis_date\": self.analysis_timestamp,\n",
        "                \"success\": True,\n",
        "                \"error\": None\n",
        "            }\n",
        "\n",
        "            return competitor_data\n",
        "\n",
        "        except Exception as e:\n",
        "            return {\n",
        "                \"company_name\": company_name or \"Unknown\",\n",
        "                \"url\": url,\n",
        "                \"scraped_data\": None,\n",
        "                \"error\": str(e),\n",
        "                \"success\": False,\n",
        "                \"analysis_date\": self.analysis_timestamp\n",
        "            }\n",
        "\n",
        "    def analyze_competitor_landscape(self, competitors: List[Dict[str, str]]) -> Dict[str, Any]:\n",
        "        \"\"\"Analyze multiple competitors and generate insights\"\"\"\n",
        "\n",
        "        print(f\"🔍 Starting competitive analysis for {len(competitors)} companies...\")\n",
        "\n",
        "        for i, competitor in enumerate(competitors, 1):\n",
        "            print(f\"📊 Analyzing {competitor['name']} ({i}/{len(competitors)})...\")\n",
        "\n",
        "            data = self.scrape_competitor_data(\n",
        "                competitor['url'],\n",
        "                competitor['name']\n",
        "            )\n",
        "            self.results.append(data)\n",
        "\n",
        "        analysis_prompt = ChatPromptTemplate.from_messages([\n",
        "            (\"system\", \"\"\"\n",
        "            You are a senior business analyst specializing in competitive intelligence.\n",
        "            Analyze the scraped competitor data and provide comprehensive insights including:\n",
        "\n",
        "            1. Market positioning analysis\n",
        "            2. Pricing strategy comparison\n",
        "            3. Feature gap analysis\n",
        "            4. Target audience overlap\n",
        "            5. Technology differentiation\n",
        "            6. Market opportunities\n",
        "            7. Competitive threats\n",
        "            8. Strategic recommendations\n",
        "\n",
        "            Provide actionable insights in JSON format with clear categories and recommendations.\n",
        "            \"\"\"),\n",
        "            (\"human\", \"Analyze this competitive data: {competitor_data}\")\n",
        "        ])\n",
        "\n",
        "        clean_data = []\n",
        "        for result in self.results:\n",
        "            if result['success']:\n",
        "                clean_data.append({\n",
        "                    'company': result['company_name'],\n",
        "                    'url': result['url'],\n",
        "                    'data': result['scraped_data']\n",
        "                })\n",
        "\n",
        "        analysis_chain = analysis_prompt | llm | JsonOutputParser()\n",
        "\n",
        "        try:\n",
        "            competitive_analysis = analysis_chain.invoke({\n",
        "                \"competitor_data\": json.dumps(clean_data, indent=2)\n",
        "            })\n",
        "        except:\n",
        "            analysis_chain_text = analysis_prompt | llm\n",
        "            competitive_analysis = analysis_chain_text.invoke({\n",
        "                \"competitor_data\": json.dumps(clean_data, indent=2)\n",
        "            })\n",
        "\n",
        "        return {\n",
        "            \"analysis\": competitive_analysis,\n",
        "            \"raw_data\": self.results,\n",
        "            \"summary_stats\": self.generate_summary_stats()\n",
        "        }\n",
        "\n",
        "    def generate_summary_stats(self) -> Dict[str, Any]:\n",
        "        \"\"\"Generate summary statistics from the analysis\"\"\"\n",
        "        successful_scrapes = sum(1 for r in self.results if r['success'])\n",
        "        failed_scrapes = len(self.results) - successful_scrapes\n",
        "\n",
        "        return {\n",
        "            \"total_companies_analyzed\": len(self.results),\n",
        "            \"successful_scrapes\": successful_scrapes,\n",
        "            \"failed_scrapes\": failed_scrapes,\n",
        "            \"success_rate\": f\"{(successful_scrapes/len(self.results)*100):.1f}%\" if self.results else \"0%\",\n",
        "            \"analysis_timestamp\": self.analysis_timestamp\n",
        "        }\n",
        "\n",
        "    def export_results(self, filename: str = None):\n",
        "        \"\"\"Export results to JSON and CSV files\"\"\"\n",
        "        if not filename:\n",
        "            filename = f\"competitive_analysis_{datetime.now().strftime('%Y%m%d_%H%M%S')}\"\n",
        "\n",
        "        with open(f\"{filename}.json\", 'w') as f:\n",
        "            json.dump({\n",
        "                \"results\": self.results,\n",
        "                \"summary\": self.generate_summary_stats()\n",
        "            }, f, indent=2)\n",
        "\n",
        "        df_data = []\n",
        "        for result in self.results:\n",
        "            if result['success']:\n",
        "                df_data.append({\n",
        "                    'Company': result['company_name'],\n",
        "                    'URL': result['url'],\n",
        "                    'Success': result['success'],\n",
        "                    'Data_Length': len(str(result['scraped_data'])) if result['scraped_data'] else 0,\n",
        "                    'Analysis_Date': result['analysis_date']\n",
        "                })\n",
        "\n",
        "        if df_data:\n",
        "            df = pd.DataFrame(df_data)\n",
        "            df.to_csv(f\"{filename}.csv\", index=False)\n",
        "\n",
        "        print(f\"✅ Results exported to {filename}.json and {filename}.csv\")"
      ],
      "metadata": {
        "id": "BC_aky-zDsnU"
      },
      "id": "BC_aky-zDsnU",
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def run_ai_saas_analysis():\n",
        "    \"\"\"Run a comprehensive analysis of AI/SaaS competitors\"\"\"\n",
        "\n",
        "    analyzer = CompetitiveAnalyzer()\n",
        "\n",
        "    ai_saas_competitors = [\n",
        "        {\"name\": \"OpenAI\", \"url\": \"https://openai.com\"},\n",
        "        {\"name\": \"Anthropic\", \"url\": \"https://anthropic.com\"},\n",
        "        {\"name\": \"Hugging Face\", \"url\": \"https://huggingface.co\"},\n",
        "        {\"name\": \"Cohere\", \"url\": \"https://cohere.ai\"},\n",
        "        {\"name\": \"Scale AI\", \"url\": \"https://scale.com\"},\n",
        "    ]\n",
        "\n",
        "    results = analyzer.analyze_competitor_landscape(ai_saas_competitors)\n",
        "\n",
        "    print(\"\\n\" + \"=\"*80)\n",
        "    print(\"🎯 COMPETITIVE ANALYSIS RESULTS\")\n",
        "    print(\"=\"*80)\n",
        "\n",
        "    print(f\"\\n📊 Summary Statistics:\")\n",
        "    stats = results['summary_stats']\n",
        "    for key, value in stats.items():\n",
        "        print(f\"   {key.replace('_', ' ').title()}: {value}\")\n",
        "\n",
        "    print(f\"\\n🔍 Strategic Analysis:\")\n",
        "    if isinstance(results['analysis'], dict):\n",
        "        for section, content in results['analysis'].items():\n",
        "            print(f\"\\n   {section.replace('_', ' ').title()}:\")\n",
        "            if isinstance(content, list):\n",
        "                for item in content:\n",
        "                    print(f\"     • {item}\")\n",
        "            else:\n",
        "                print(f\"     {content}\")\n",
        "    else:\n",
        "        print(results['analysis'])\n",
        "\n",
        "    analyzer.export_results(\"ai_saas_competitive_analysis\")\n",
        "\n",
        "    return results"
      ],
      "metadata": {
        "id": "FvjwvDPhD9vj"
      },
      "id": "FvjwvDPhD9vj",
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def run_ecommerce_analysis():\n",
        "    \"\"\"Analyze e-commerce platform competitors\"\"\"\n",
        "\n",
        "    analyzer = CompetitiveAnalyzer()\n",
        "\n",
        "    ecommerce_competitors = [\n",
        "        {\"name\": \"Shopify\", \"url\": \"https://shopify.com\"},\n",
        "        {\"name\": \"WooCommerce\", \"url\": \"https://woocommerce.com\"},\n",
        "        {\"name\": \"BigCommerce\", \"url\": \"https://bigcommerce.com\"},\n",
        "        {\"name\": \"Magento\", \"url\": \"https://magento.com\"},\n",
        "    ]\n",
        "\n",
        "    results = analyzer.analyze_competitor_landscape(ecommerce_competitors)\n",
        "    analyzer.export_results(\"ecommerce_competitive_analysis\")\n",
        "\n",
        "    return results"
      ],
      "metadata": {
        "id": "SRNDY1HKEC8y"
      },
      "id": "SRNDY1HKEC8y",
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "@chain\n",
        "def social_media_monitoring_chain(company_urls: List[str], config: RunnableConfig):\n",
        "    \"\"\"Monitor social media presence and engagement strategies of competitors\"\"\"\n",
        "\n",
        "    social_media_prompt = ChatPromptTemplate.from_messages([\n",
        "        (\"system\", \"\"\"\n",
        "        You are a social media strategist. Analyze the social media presence and strategies\n",
        "        of these companies. Focus on:\n",
        "        1. Platform presence (LinkedIn, Twitter, Instagram, etc.)\n",
        "        2. Content strategy patterns\n",
        "        3. Engagement tactics\n",
        "        4. Community building approaches\n",
        "        5. Brand voice and messaging\n",
        "        6. Posting frequency and timing\n",
        "        Provide actionable insights for improving social media strategy.\n",
        "        \"\"\"),\n",
        "        (\"human\", \"Analyze social media data for: {urls}\")\n",
        "    ])\n",
        "\n",
        "    social_data = []\n",
        "    for url in company_urls:\n",
        "        try:\n",
        "            result = smartscraper.invoke({\n",
        "                \"user_prompt\": \"Extract all social media links, community engagement features, and social proof elements\",\n",
        "                \"website_url\": url,\n",
        "            })\n",
        "            social_data.append({\"url\": url, \"social_data\": result})\n",
        "        except Exception as e:\n",
        "            social_data.append({\"url\": url, \"error\": str(e)})\n",
        "\n",
        "    chain = social_media_prompt | llm\n",
        "    analysis = chain.invoke({\"urls\": json.dumps(social_data, indent=2)}, config=config)\n",
        "\n",
        "    return {\n",
        "        \"social_analysis\": analysis,\n",
        "        \"raw_social_data\": social_data\n",
        "    }"
      ],
      "metadata": {
        "id": "xU_kOmRvEDbk"
      },
      "id": "xU_kOmRvEDbk",
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def check_credits():\n",
        "    \"\"\"Check available credits\"\"\"\n",
        "    try:\n",
        "        credits_info = credits.invoke({})\n",
        "        print(f\"💳 Available Credits: {credits_info}\")\n",
        "        return credits_info\n",
        "    except Exception as e:\n",
        "        print(f\"⚠️  Could not check credits: {e}\")\n",
        "        return None"
      ],
      "metadata": {
        "id": "8kfFZ6cgEIoD"
      },
      "id": "8kfFZ6cgEIoD",
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "if __name__ == \"__main__\":\n",
        "    print(\"🚀 Advanced Competitive Analysis Tool with Gemini AI\")\n",
        "    print(\"=\"*60)\n",
        "\n",
        "    check_credits()\n",
        "\n",
        "    print(\"\\n🤖 Running AI/SaaS Competitive Analysis...\")\n",
        "    ai_results = run_ai_saas_analysis()\n",
        "\n",
        "    run_additional = input(\"\\n❓ Run e-commerce analysis as well? (y/n): \").lower().strip()\n",
        "    if run_additional == 'y':\n",
        "        print(\"\\n🛒 Running E-commerce Platform Analysis...\")\n",
        "        ecom_results = run_ecommerce_analysis()\n",
        "\n",
        "    print(\"\\n✨ Analysis complete! Check the exported files for detailed results.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OoKzn0N6Crwr",
        "outputId": "7e0639ef-f62c-42e9-b0f1-ed4436fde364"
      },
      "id": "OoKzn0N6Crwr",
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Google API key for Gemini:\n",
            "··········\n",
            "🚀 Advanced Competitive Analysis Tool with Gemini AI\n",
            "============================================================\n",
            "💳 Available Credits: {'remaining_credits': 8, 'total_credits_used': 42}\n",
            "\n",
            "🤖 Running AI/SaaS Competitive Analysis...\n",
            "🔍 Starting competitive analysis for 5 companies...\n",
            "📊 Analyzing OpenAI (1/5)...\n",
            "📊 Analyzing Anthropic (2/5)...\n",
            "📊 Analyzing Hugging Face (3/5)...\n",
            "📊 Analyzing Cohere (4/5)...\n",
            "📊 Analyzing Scale AI (5/5)...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.11/dist-packages/langchain_google_genai/chat_models.py:424: UserWarning: Convert_system_message_to_human will be deprecated!\n",
            "  warnings.warn(\"Convert_system_message_to_human will be deprecated!\")\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "================================================================================\n",
            "🎯 COMPETITIVE ANALYSIS RESULTS\n",
            "================================================================================\n",
            "\n",
            "📊 Summary Statistics:\n",
            "   Total Companies Analyzed: 5\n",
            "   Successful Scrapes: 0\n",
            "   Failed Scrapes: 5\n",
            "   Success Rate: 0.0%\n",
            "   Analysis Timestamp: 2025-06-02 14:29:15\n",
            "\n",
            "🔍 Strategic Analysis:\n",
            "\n",
            "   Analysis:\n",
            "     {'competitorData': [], 'insights': {'marketPositioning': {'summary': 'No competitor data provided.  Unable to perform market positioning analysis.  Requires competitor data including brand positioning statements, target market descriptions, and perceived value propositions.', 'recommendations': ['Gather competitor data on brand positioning, target market, and value proposition.  Conduct surveys and market research to understand customer perceptions.']}, 'pricingStrategy': {'summary': 'No competitor data provided. Unable to compare pricing strategies. Requires competitor pricing data for various products/services.', 'recommendations': ['Gather competitor pricing data for comparable offerings. Analyze pricing models (value-based, cost-plus, competitive). Identify pricing trends and patterns.']}, 'featureGapAnalysis': {'summary': 'No competitor data provided. Unable to analyze feature gaps. Requires a feature list for our product and competitor products.', 'recommendations': ['Create a comprehensive feature list for our product and key competitors.  Identify gaps and opportunities for differentiation and improvement.']}, 'targetAudienceOverlap': {'summary': 'No competitor data provided. Unable to assess target audience overlap. Requires data on competitor target demographics, psychographics, and buying behavior.', 'recommendations': [\"Conduct market research to identify the demographics, psychographics, and buying behavior of our target audience and our competitors'.  Use tools like surveys, focus groups, and customer relationship management (CRM) data.\"]}, 'technologyDifferentiation': {'summary': 'No competitor data provided. Unable to analyze technology differentiation. Requires information on competitor technologies, platforms, and infrastructure.', 'recommendations': ['Research competitor technologies, patents, and infrastructure. Identify areas where our technology offers a competitive advantage or where improvements are needed.']}, 'marketOpportunities': {'summary': 'No competitor data provided. Unable to identify market opportunities. Requires market size, growth rate, and competitor market share data.', 'recommendations': ['Conduct market research to assess market size, growth rate, and competitor market share. Identify underserved segments and unmet needs.']}, 'competitiveThreats': {'summary': 'No competitor data provided. Unable to identify competitive threats. Requires data on competitor strengths, weaknesses, strategies, and potential disruptive innovations.', 'recommendations': ['Monitor competitor activities, including new product launches, marketing campaigns, and strategic partnerships.  Identify potential threats and develop mitigation strategies.']}, 'strategicRecommendations': {'summary': 'Without competitor data, strategic recommendations are impossible.  Data collection is the first priority.', 'recommendations': ['Prioritize gathering comprehensive competitor data.  Develop a competitive intelligence program to continuously monitor the competitive landscape.  Use this data to inform product development, marketing, and sales strategies.']}}}\n",
            "✅ Results exported to ai_saas_competitive_analysis.json and ai_saas_competitive_analysis.csv\n",
            "\n",
            "❓ Run e-commerce analysis as well? (y/n): n\n",
            "\n",
            "✨ Analysis complete! Check the exported files for detailed results.\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.9"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}